// vim:ts=5:sw=5:

#include <iostream>
#include <fstream>
#include <algorithm>
#include <vector>
#include <boost/functional/hash.hpp>
#include <google/sparse_hash_map>
#include "client/dbclient.h"

using namespace mongo;
using namespace std;

typedef google::sparse_hash_map<string, int, boost::hash<string> > Dict;
typedef Dict::value_type Item;

bool compareWordCount(const Item *a, const Item *b)
{
     return a->second > b->second;
}

int copyCollection(const string &from, const string &to, const char *json=NULL)
{
     int count = 0;
     BSONObj query = (json == NULL ? BSONObj() : fromjson(json));
     DBClientConnection conn;
     conn.connect("192.168.1.114:27017"); 
     auto_ptr<DBClientCursor> cursor = conn.query(from, query);
     while( cursor->more() )
     {
          BSONObj doc = cursor->next();
          conn.insert(to,doc);
          count++;
          if (count % 1000 == 0)
               cout << count << " copied" << endl;
     }
     cout << "total " << count << endl;
     return count;
}
void word_count(Dict &words, const string &ns, const char *field)
{
     int n = 0;
     DBClientConnection conn;
     conn.connect("192.168.1.114:27017");
     BSONObj fs = BSON(field << 1);
     auto_ptr<DBClientCursor> cursor = conn.query(ns, BSONObj(), 0,0, &fs);
     while( cursor->more() )
     {
          const char *atext = (cursor->next()).getStringField(field);
          words[string(atext)]++;
          if (++n % 100000 == 0)
          {
               cout << n << " words:" << words.size() << endl;
          }
     }
     cout << "count over:" << words.size() << " words" << endl;
}

int main(int args, char **argv)
{
     ofstream out;
     if (args > 1)
          out.open(argv[1]);
     Dict words;
     word_count(words,"zbot_development.download","referer_atext");
     vector<Item*> vec;
     vec.reserve(words.size());
     for(Dict::iterator i = words.begin(); i != words.end(); i++)
     {
          vec.push_back(&(*i));
     }
     sort(vec.begin(), vec.end(), compareWordCount);
     for(vector<Item*>::iterator i = vec.begin(); i != vec.end(); i++)
     {
          cout << (*i)->first << " : " << (*i)->second << endl;
          if (out.is_open())
               out << (*i)->first << " : " << (*i)->second << endl;
     }
     return 0;
}
